package com.zhiletu.collect.util;

import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;

import javax.script.ScriptException;

import org.apache.http.HeaderElement;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.ParseException;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.params.HttpProtocolParams;
import org.apache.http.protocol.HTTP;
import org.apache.http.util.CharArrayBuffer;

/**
 *
 * @Title: 网页操作工具类
 * @Package com.zhiletu.collect.util
 * @Description: 下载网页工具
 * @author zhiletu.com
 * @date 2021年3月11日
 * @version V1.0
 */
public class HtmlUtil {

//	public static void main(String[] args){
//		String url = "http://news.sina.com.cn/c/2015-08-28/112232247575.shtml?cre=chinapc&mod=inf&loc=1&r=h&rfunc=9";
//		String charest = "utf-8";
//		try {
//			new HtmlUtil().doGet(url, charest);
//		} catch (Exception e) { 
//			e.printStackTrace();
//		}
//	}

	/**
	 * Get the entity content as a String, using the provided default character set
	 * if none is found in the entity.
	 * If defaultCharset is null, the default "ISO-8859-1" is used.
	 *
	 * @param entity must not be null
	 * @param defaultCharset character set to be applied if none found in the entity
	 * @return the entity content as a String
	 * @throws ParseException if header elements cannot be parsed
	 * @throws IllegalArgumentException if entity is null or if content length > Integer.MAX_VALUE
	 * @throws IOException if an error occurs reading the input stream
	 */
	public static String toString(
			final HttpEntity entity, final String defaultCharset) throws IOException, ParseException {
		if (entity == null) {
			throw new IllegalArgumentException("HTTP entity may not be null");
		}
		InputStream instream = entity.getContent();
		if (instream == null) {
			return "";
		}
		if (entity.getContentLength() > Integer.MAX_VALUE) {
			throw new IllegalArgumentException("HTTP entity too large to be buffered in memory");
		}
		int i = (int) entity.getContentLength();
		if (i < 0) {
			i = 4096;
		}
		//获取请求中的编码
		String charset = getContentCharSet(entity);
		//若没返回编码则使用传递的编码
		if (charset == null) {
			charset = defaultCharset;
		}
		if (charset == null) {
			charset = HTTP.DEFAULT_CONTENT_CHARSET;
		}
		Reader reader = new InputStreamReader(instream, charset);
		CharArrayBuffer buffer = new CharArrayBuffer(i);
		try {
			char[] tmp = new char[1024];
			int l;
			while ((l = reader.read(tmp)) != -1) {
				buffer.append(tmp, 0, l);
			}
		}
		finally {
			reader.close();
		}
		return buffer.toString();
	}

	/**
	 * Obtains character set of the entity, if known.
	 * <meta charset="utf-8"/>
	 * <meta charset="gbk"/>
	 * @param entity must not be null
	 * @return the character set, or null if not found
	 * @throws ParseException if header elements cannot be parsed
	 * @throws IOException
	 * @throws IllegalStateException
	 * @throws IllegalArgumentException if entity is null
	 */
	public static String getContentCharSet(final HttpEntity entity)
			throws ParseException, IllegalStateException, IOException {

		if (entity == null) {
			throw new IllegalArgumentException("HTTP entity may not be null");
		}
		String charset = null;
		// 老子先根据dom判断字符集
		if (entity.getContentType() != null) {
			HeaderElement values[] = entity.getContentType().getElements();
			if (values.length > 0) {
				NameValuePair param = values[0].getParameterByName("charset");
				if (param != null) {
					charset = param.getValue();
				}
			}
		}
		// dom解析失败，使用必杀技：输入流获取字符集
		if (charset == null) {
			charset = new EncodeParseUtil().getInputStreamEncode(entity.getContent(), false);
			//System.out.println("分析字符集为："+charset);
		}
		return charset;
	}

	/**
	 * 将抓取到的html网页解析出内容（取掉html标签除了p标签）
	 * @param htmlSource
	 * @return
	 */
	public static String parseHtmlToText(String htmlSource) {
		/*去掉所有的html标记(除了p标签)*/
		String delHtmlTag = "function delHtmlTag(str){str = str.replace(/<script[^>]*?>[\\s\\S]*?<\\/script>/g,\"\"); str = str.replace(/<style[^>]*?>[\\s\\S]*?<\\/style>/g,\"\"); return str.replace(/<(?!\\/?[pP])[^>]+>/g,\"\");}";

		String content = "";
		try {
			content = ScriptingAPI.invokeScript(delHtmlTag, "delHtmlTag", new Object[] { htmlSource });
		}
		catch (ScriptException e) {
			throw new RuntimeException(e);
		}
		catch (NoSuchMethodException e) {
			throw new RuntimeException(e);
		}

		return content;
	}

	/**
	 * 将抓取到的html网页解析出内容（取掉html标签,包括p标签）
	 * @param htmlSource
	 * @return
	 */
	public static String parseHtmlPToText(String htmlSource) {
		/*去掉所有的html标记(除了p标签)*/
		String delHtmlTag = "function delHtmlTag(str){str = str.replace(/<script[^>]*?>[\\s\\S]*?<\\/script>/g,\"\"); str = str.replace(/<style[^>]*?>[\\s\\S]*?<\\/style>/g,\"\"); return str.replace(/<[^>]+>/g,\"\");}";

		String content = "";
		try {
			content = ScriptingAPI.invokeScript(delHtmlTag, "delHtmlTag", new Object[] { htmlSource });
		}
		catch (ScriptException e) {
			throw new RuntimeException(e);
		}
		catch (NoSuchMethodException e) {
			throw new RuntimeException(e);
		}

		return content;
	}

	/**
	 * 处理GET请求，返回整个页面
	 *
	 * @param url 访问地址
	 * @param params 编码参数
	 * @return
	 * @throws Exception
	 * @throws Exception
	 */
	public synchronized String doGet(String userAgent, String url, String... params) throws Exception {
		DefaultHttpClient httpclient = new DefaultHttpClient();
		HttpProtocolParams.setUserAgent(httpclient.getParams(), userAgent);
		//"Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.1.9) Gecko/20100315 Firefox/3.5.9");
		String charset = "UTF-8";
		if (null != params && params.length >= 1) {
			charset = params[0];
		}
		HttpGet httpget = new HttpGet();
		String content = "";
		httpget.setURI(new java.net.URI(url));
		HttpResponse response = httpclient.execute(httpget);
		HttpEntity entity = response.getEntity();
		if (entity != null) {
			//使用EntityUtils的toString方法，传递默认编码，在EntityUtils中的默认编码是ISO-8859-1
			content = toString(entity, charset);
			httpget.abort();
			httpclient.getConnectionManager().shutdown();
		}
		//System.out.println(content);
		return content;
	}
}
